function [AUC,VAL]=cvroc(obs,pred,plt,salto)

% ROC Response Operating Characteristic curves
% Syntax: AUC=roc(obs,pred,{plt},{salto})
% Description:
%    Creates a ROC curve plot of sensitivity vs 1-specificity
%    which is equivalent of p(true positives) vs p(false negatives)
%    
%    Input Arguments:
%
%    obs - observed outcome
%    pred - predicted outcome
%    plt:
%        1 - regular ROC curve plot
%        2 - ROC curve with threshold values
%        any other option will cause the graphic
%        display not to be produced.
%
%    Output arguments
%
%    AUC - area under the curve
%    VAL - values definign the ROC curve 
%
% Jonas Almeida, May 2006

if nargin<3;plt=0;end % By default do not plot
if nargin<4;salto=unique(pred)';end

obs_pos=sum(obs==1); % Observed positives = 100% true positives
positivos=pred(find(obs==1));npos=length(positivos);
negativos=pred(find(obs==0));nneg=length(negativos);

VAL=[]; 
for tr=salto    % tr= threshold
    VAL=[VAL;[tr,sum(negativos>=tr),sum(positivos>=tr)]];
end
VAL(:,2)=VAL(:,2)./nneg;
VAL(:,3)=VAL(:,3)./npos;
% inverting order
VAL=VAL(end:-1:1,:);

% Area under the curve:
arestas=[VAL(1:end-1,2),VAL(2:end,2),VAL(1:end-1,3),VAL(2:end,3)]; % edges
AUC=sum((arestas(:,2)-arestas(:,1)).*(arestas(:,3)+(arestas(:,4)-arestas(:,3))./2)); %area under the ROC curve

%Graphic display of ROC curve
switch plt
case 1
    h=area(VAL(:,2),VAL(:,3),'FaceColor','interp');
    set(h,'FaceColor',[0.75 0.75 0.75])
    G=get(h);set(G.Parent,'FontName','Verdana','FontSize',8);
    hold on
    plot([0;1],[0;1],':');
    ylabel('sensitivity','FontName','Verdana','FontSize',8)
    xlabel('1-specificity','FontName','Verdana','FontSize',8)
    text(0.6,0.4,[' Area = \newline',num2str(AUC)],'FontSize',8,'FontName','Verdana')
    gridover
    hold off
    
case 2 % plot threshold value too
    subplot(2,1,1)
    h=area(VAL(:,2),VAL(:,3));
    set(h,'FaceColor',[0.75 0.75 0.75])
    ylabel('sensitivity\newline(fraction of true positives)')
    xlabel('1-specificity (fraction of false positives)')
    text(0.6,0.4,[' Area = ',num2str(AUC)],'FontSize',14)
    grid on
    hold on
    plot([0;1],[0;1],':');
    hold off
    subplot(2,1,2)
    plot(VAL(:,2),salto(end:-1:1)','LineWidth',3,'Color',[0.5 0.5 0.5])
    G=get(gcf);
    set(G.Children(1),'XAxisLocation','top')
    grid on
    ylabel('segmentation value \newline(decision threshold)')
    
case 3
    h=area(VAL(:,2),VAL(:,3),'FaceColor','interp');
    set(h,'FaceColor',[0.75 0.75 0.75])
    G=get(h);set(G.Parent,'FontName','Verdana','FontSize',8,'XTick',[0:0.2:1],'YAxisLocation','right','XLim',[0 1],'YLim',[0 1]);
    hold on
    plot([0;1],[0;1],':','Color',[0.25 0.25 0.25])
    plot(VAL(:,2),salto(end:-1:1)');
    ylabel('sensitivity','FontName','Verdana','FontSize',8,'Rotation',270,'Position',[1.35 0.494595 1.00011])
    xlabel('1-specificity','FontName','Verdana','FontSize',8)
    gridover(G.Parent)
    text(0.5,0.4,[' Area = \newline',num2str(AUC)],'FontSize',8,'FontName','Verdana')
    hold off 
    title('Cross Val. ROC')
end


% ----- call in functions -----
function gridover(h) % grid overlay
G1=get(h);%G1=get(G.Children(1));
%set(h,'XTick',G1.YTick); % <-- to make X and Y axis equal
ax=axis;
for i=2:length(G1.XTick)-1 % X grid
    plot([G1.XTick(i),G1.XTick(i)],[ax(3),ax(4)],':','Color',[0.5 0.5 0.5])
end
for i=2:length(G1.YTick)-1 % X grid
    plot([ax(1),ax(2)],[G1.YTick(i),G1.YTick(i)],':','Color',[0.5 0.5 0.5])
end
        